import numpy as np
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import pandas as pd
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
import plotly.graph_objects as go
import math
import seaborn as sns
from sklearn.metrics import mean_squared_error
np.random.seed(1)
tf.random.set_seed(1)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, Dropout, RepeatVector, TimeDistributed
from keras import backend
MODELFILENAME = 'MODELS/DNN_1h_TFM'
TIME_STEPS=6 #1h
UNITS=22
DROPOUT=0.779
ACTIVATION='sigmoid'
OPTIMIZER='adamax'
EPOCHS=85
BATCHSIZE=23
VALIDATIONSPLIT=0.2
# Code to read csv file into Colaboratory:
# from google.colab import files
# uploaded = files.upload()
# import io
# df = pd.read_csv(io.BytesIO(uploaded['SentDATA.csv']))
# Dataset is now stored in a Pandas Dataframe
df = pd.read_csv('../../data/dadesTFM.csv')
df.reset_index(inplace=True)
df['Time'] = pd.to_datetime(df['Time'])
df = df.set_index('Time')
columns = ['PM1','PM25','PM10','PM1ATM','PM25ATM','PM10ATM']
df1 = df.copy();
df1 = df1.rename(columns={"PM 1":"PM1","PM 2.5":"PM25","PM 10":"PM10","PM 1 ATM":"PM1ATM","PM 2.5 ATM":"PM25ATM","PM 10 ATM":"PM10ATM"})
df1['PM1'] = df['PM 1'].astype(np.float32)
df1['PM25'] = df['PM 2.5'].astype(np.float32)
df1['PM10'] = df['PM 10'].astype(np.float32)
df1['PM1ATM'] = df['PM 1 ATM'].astype(np.float32)
df1['PM25ATM'] = df['PM 2.5 ATM'].astype(np.float32)
df1['PM10ATM'] = df['PM 10 ATM'].astype(np.float32)
df2 = df1.copy()
train_size = int(len(df2) * 0.8)
test_size = len(df2) - train_size
train, test = df2.iloc[0:train_size], df2.iloc[train_size:len(df2)]
train.shape, test.shape
((3117, 7), (780, 7))
#Standardize the data
for col in columns:
scaler = StandardScaler()
train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]])
def create_sequences(X, y, time_steps=TIME_STEPS):
Xs, ys = [], []
for i in range(len(X)-time_steps):
Xs.append(X.iloc[i:(i+time_steps)].values)
ys.append(y.iloc[i+time_steps])
return np.array(Xs), np.array(ys)
X_train, y_train = create_sequences(train[[columns[1]]], train[columns[1]])
#X_test, y_test = create_sequences(test[[columns[1]]], test[columns[1]])
print(f'X_train shape: {X_train.shape}')
print(f'y_train shape: {y_train.shape}')
X_train shape: (3111, 6, 1) y_train shape: (3111,)
#afegir nova mètrica
def rmse(y_true, y_pred):
return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))
model = Sequential()
model.add(Dense(units=UNITS, input_shape=(X_train.shape[1], X_train.shape[2]), activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dropout(rate=DROPOUT))
model.add(Dense(X_train.shape[2],activation=ACTIVATION))
model.compile(optimizer=OPTIMIZER, loss='mae',metrics=[rmse,'mse'])
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= dense (Dense) (None, 6, 22) 44 _________________________________________________________________ dense_1 (Dense) (None, 6, 16) 368 _________________________________________________________________ dropout (Dropout) (None, 6, 16) 0 _________________________________________________________________ dense_2 (Dense) (None, 6, 1) 17 ================================================================= Total params: 429 Trainable params: 429 Non-trainable params: 0 _________________________________________________________________
history = model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCHSIZE, validation_split=VALIDATIONSPLIT,
callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')], shuffle=False)
Epoch 1/85 109/109 [==============================] - 0s 3ms/step - loss: 0.8471 - rmse: 0.8511 - mse: 1.0046 - val_loss: 1.2447 - val_rmse: 1.2452 - val_mse: 1.7497 Epoch 2/85 109/109 [==============================] - 0s 1ms/step - loss: 0.8075 - rmse: 0.8138 - mse: 0.9258 - val_loss: 1.1745 - val_rmse: 1.1755 - val_mse: 1.5746 Epoch 3/85 109/109 [==============================] - 0s 1ms/step - loss: 0.7764 - rmse: 0.7863 - mse: 0.8701 - val_loss: 1.1100 - val_rmse: 1.1114 - val_mse: 1.4234 Epoch 4/85 109/109 [==============================] - 0s 988us/step - loss: 0.7469 - rmse: 0.7614 - mse: 0.8214 - val_loss: 1.0525 - val_rmse: 1.0544 - val_mse: 1.2957 Epoch 5/85 109/109 [==============================] - 0s 1ms/step - loss: 0.7284 - rmse: 0.7467 - mse: 0.7922 - val_loss: 1.0054 - val_rmse: 1.0078 - val_mse: 1.1965 Epoch 6/85 109/109 [==============================] - 0s 1ms/step - loss: 0.7110 - rmse: 0.7331 - mse: 0.7666 - val_loss: 0.9705 - val_rmse: 0.9733 - val_mse: 1.1260 Epoch 7/85 109/109 [==============================] - 0s 1ms/step - loss: 0.7012 - rmse: 0.7254 - mse: 0.7509 - val_loss: 0.9456 - val_rmse: 0.9488 - val_mse: 1.0777 Epoch 8/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6908 - rmse: 0.7170 - mse: 0.7335 - val_loss: 0.9281 - val_rmse: 0.9316 - val_mse: 1.0447 Epoch 9/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6834 - rmse: 0.7105 - mse: 0.7221 - val_loss: 0.9155 - val_rmse: 0.9192 - val_mse: 1.0215 Epoch 10/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6799 - rmse: 0.7080 - mse: 0.7154 - val_loss: 0.9067 - val_rmse: 0.9107 - val_mse: 1.0058 Epoch 11/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6756 - rmse: 0.7046 - mse: 0.7102 - val_loss: 0.9000 - val_rmse: 0.9042 - val_mse: 0.9938 Epoch 12/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6691 - rmse: 0.6984 - mse: 0.7001 - val_loss: 0.8950 - val_rmse: 0.8994 - val_mse: 0.9851 Epoch 13/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6665 - rmse: 0.6964 - mse: 0.6950 - val_loss: 0.8906 - val_rmse: 0.8950 - val_mse: 0.9775 Epoch 14/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6612 - rmse: 0.6910 - mse: 0.6874 - val_loss: 0.8874 - val_rmse: 0.8919 - val_mse: 0.9720 Epoch 15/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6589 - rmse: 0.6886 - mse: 0.6802 - val_loss: 0.8847 - val_rmse: 0.8893 - val_mse: 0.9675 Epoch 16/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6522 - rmse: 0.6816 - mse: 0.6680 - val_loss: 0.8821 - val_rmse: 0.8867 - val_mse: 0.9630 Epoch 17/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6491 - rmse: 0.6780 - mse: 0.6645 - val_loss: 0.8798 - val_rmse: 0.8845 - val_mse: 0.9593 Epoch 18/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6434 - rmse: 0.6726 - mse: 0.6562 - val_loss: 0.8780 - val_rmse: 0.8827 - val_mse: 0.9563 Epoch 19/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6378 - rmse: 0.6673 - mse: 0.6496 - val_loss: 0.8768 - val_rmse: 0.8816 - val_mse: 0.9543 Epoch 20/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6350 - rmse: 0.6644 - mse: 0.6439 - val_loss: 0.8757 - val_rmse: 0.8805 - val_mse: 0.9525 Epoch 21/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6305 - rmse: 0.6598 - mse: 0.6365 - val_loss: 0.8746 - val_rmse: 0.8794 - val_mse: 0.9507 Epoch 22/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6332 - rmse: 0.6629 - mse: 0.6400 - val_loss: 0.8738 - val_rmse: 0.8786 - val_mse: 0.9495 Epoch 23/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6294 - rmse: 0.6595 - mse: 0.6375 - val_loss: 0.8732 - val_rmse: 0.8780 - val_mse: 0.9485 Epoch 24/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6267 - rmse: 0.6567 - mse: 0.6316 - val_loss: 0.8726 - val_rmse: 0.8775 - val_mse: 0.9477 Epoch 25/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6283 - rmse: 0.6591 - mse: 0.6362 - val_loss: 0.8723 - val_rmse: 0.8772 - val_mse: 0.9472 Epoch 26/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6229 - rmse: 0.6521 - mse: 0.6234 - val_loss: 0.8720 - val_rmse: 0.8769 - val_mse: 0.9467 Epoch 27/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6232 - rmse: 0.6529 - mse: 0.6263 - val_loss: 0.8716 - val_rmse: 0.8765 - val_mse: 0.9461 Epoch 28/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6211 - rmse: 0.6512 - mse: 0.6251 - val_loss: 0.8713 - val_rmse: 0.8762 - val_mse: 0.9457 Epoch 29/85 109/109 [==============================] - 0s 2ms/step - loss: 0.6184 - rmse: 0.6483 - mse: 0.6210 - val_loss: 0.8713 - val_rmse: 0.8762 - val_mse: 0.9456 Epoch 30/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6177 - rmse: 0.6474 - mse: 0.6216 - val_loss: 0.8710 - val_rmse: 0.8759 - val_mse: 0.9452 Epoch 31/85 109/109 [==============================] - 0s 988us/step - loss: 0.6187 - rmse: 0.6490 - mse: 0.6214 - val_loss: 0.8709 - val_rmse: 0.8758 - val_mse: 0.9449 Epoch 32/85 109/109 [==============================] - 0s 997us/step - loss: 0.6146 - rmse: 0.6438 - mse: 0.6136 - val_loss: 0.8708 - val_rmse: 0.8757 - val_mse: 0.9447 Epoch 33/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6152 - rmse: 0.6449 - mse: 0.6173 - val_loss: 0.8707 - val_rmse: 0.8756 - val_mse: 0.9445 Epoch 34/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6135 - rmse: 0.6417 - mse: 0.6147 - val_loss: 0.8704 - val_rmse: 0.8753 - val_mse: 0.9442 Epoch 35/85 109/109 [==============================] - 0s 988us/step - loss: 0.6157 - rmse: 0.6453 - mse: 0.6203 - val_loss: 0.8704 - val_rmse: 0.8753 - val_mse: 0.9441 Epoch 36/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6137 - rmse: 0.6434 - mse: 0.6159 - val_loss: 0.8704 - val_rmse: 0.8753 - val_mse: 0.9440 Epoch 37/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6110 - rmse: 0.6401 - mse: 0.6140 - val_loss: 0.8703 - val_rmse: 0.8753 - val_mse: 0.9439 Epoch 38/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6101 - rmse: 0.6392 - mse: 0.6074 - val_loss: 0.8701 - val_rmse: 0.8751 - val_mse: 0.9437 Epoch 39/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6087 - rmse: 0.6379 - mse: 0.6040 - val_loss: 0.8701 - val_rmse: 0.8751 - val_mse: 0.9436 Epoch 40/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6103 - rmse: 0.6394 - mse: 0.6107 - val_loss: 0.8700 - val_rmse: 0.8750 - val_mse: 0.9434 Epoch 41/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6065 - rmse: 0.6353 - mse: 0.6036 - val_loss: 0.8698 - val_rmse: 0.8748 - val_mse: 0.9431 Epoch 42/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6059 - rmse: 0.6352 - mse: 0.6030 - val_loss: 0.8696 - val_rmse: 0.8746 - val_mse: 0.9429 Epoch 43/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6027 - rmse: 0.6316 - mse: 0.5983 - val_loss: 0.8695 - val_rmse: 0.8745 - val_mse: 0.9427 Epoch 44/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6049 - rmse: 0.6344 - mse: 0.6034 - val_loss: 0.8693 - val_rmse: 0.8743 - val_mse: 0.9424 Epoch 45/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6030 - rmse: 0.6318 - mse: 0.5991 - val_loss: 0.8692 - val_rmse: 0.8741 - val_mse: 0.9423 Epoch 46/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6039 - rmse: 0.6330 - mse: 0.6028 - val_loss: 0.8692 - val_rmse: 0.8741 - val_mse: 0.9423 Epoch 47/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6031 - rmse: 0.6325 - mse: 0.6009 - val_loss: 0.8691 - val_rmse: 0.8741 - val_mse: 0.9422 Epoch 48/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6006 - rmse: 0.6294 - mse: 0.5968 - val_loss: 0.8690 - val_rmse: 0.8740 - val_mse: 0.9421 Epoch 49/85 109/109 [==============================] - 0s 1ms/step - loss: 0.6008 - rmse: 0.6299 - mse: 0.6004 - val_loss: 0.8689 - val_rmse: 0.8739 - val_mse: 0.9420 Epoch 50/85 109/109 [==============================] - 0s 2ms/step - loss: 0.5997 - rmse: 0.6287 - mse: 0.5937 - val_loss: 0.8689 - val_rmse: 0.8739 - val_mse: 0.9419 Epoch 51/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5982 - rmse: 0.6270 - mse: 0.5936 - val_loss: 0.8687 - val_rmse: 0.8736 - val_mse: 0.9416 Epoch 52/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5953 - rmse: 0.6237 - mse: 0.5906 - val_loss: 0.8686 - val_rmse: 0.8735 - val_mse: 0.9415 Epoch 53/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5962 - rmse: 0.6250 - mse: 0.5920 - val_loss: 0.8685 - val_rmse: 0.8734 - val_mse: 0.9414 Epoch 54/85 109/109 [==============================] - 0s 2ms/step - loss: 0.5966 - rmse: 0.6249 - mse: 0.5964 - val_loss: 0.8685 - val_rmse: 0.8735 - val_mse: 0.9413 Epoch 55/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5948 - rmse: 0.6230 - mse: 0.5888 - val_loss: 0.8685 - val_rmse: 0.8735 - val_mse: 0.9413 Epoch 56/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5941 - rmse: 0.6235 - mse: 0.5887 - val_loss: 0.8685 - val_rmse: 0.8735 - val_mse: 0.9413 Epoch 57/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5942 - rmse: 0.6238 - mse: 0.5888 - val_loss: 0.8684 - val_rmse: 0.8734 - val_mse: 0.9412 Epoch 58/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5898 - rmse: 0.6185 - mse: 0.5800 - val_loss: 0.8684 - val_rmse: 0.8733 - val_mse: 0.9412 Epoch 59/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5947 - rmse: 0.6235 - mse: 0.5893 - val_loss: 0.8683 - val_rmse: 0.8733 - val_mse: 0.9411 Epoch 60/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5905 - rmse: 0.6192 - mse: 0.5823 - val_loss: 0.8683 - val_rmse: 0.8733 - val_mse: 0.9411 Epoch 61/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5929 - rmse: 0.6218 - mse: 0.5903 - val_loss: 0.8683 - val_rmse: 0.8733 - val_mse: 0.9411 Epoch 62/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5942 - rmse: 0.6234 - mse: 0.5897 - val_loss: 0.8684 - val_rmse: 0.8734 - val_mse: 0.9412 Epoch 63/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5926 - rmse: 0.6219 - mse: 0.5904 - val_loss: 0.8684 - val_rmse: 0.8734 - val_mse: 0.9412 Epoch 64/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5893 - rmse: 0.6185 - mse: 0.5845 - val_loss: 0.8684 - val_rmse: 0.8734 - val_mse: 0.9412 Epoch 65/85 109/109 [==============================] - 0s 1ms/step - loss: 0.5921 - rmse: 0.6213 - mse: 0.5853 - val_loss: 0.8684 - val_rmse: 0.8734 - val_mse: 0.9412
import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='MAE Training loss')
plt.plot(history.history['val_loss'], label='MAE Validation loss')
plt.plot(history.history['mse'], label='MSE Training loss')
plt.plot(history.history['val_mse'], label='MSE Validation loss')
plt.plot(history.history['rmse'], label='RMSE Training loss')
plt.plot(history.history['val_rmse'], label='RMSE Validation loss')
plt.legend();
X_train_pred = model.predict(X_train, verbose=0)
train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)
plt.hist(train_mae_loss, bins=50)
plt.xlabel('Train MAE loss')
plt.ylabel('Number of Samples');
def evaluate_prediction(predictions, actual, model_name):
errors = predictions - actual
mse = np.square(errors).mean()
rmse = np.sqrt(mse)
mae = np.abs(errors).mean()
print(model_name + ':')
print('Mean Absolute Error: {:.4f}'.format(mae))
print('Root Mean Square Error: {:.4f}'.format(rmse))
print('Mean Square Error: {:.4f}'.format(mse))
print('')
return mae,rmse,mse
mae,rmse,mse = evaluate_prediction(X_train_pred, X_train,"LSTM")
LSTM: Mean Absolute Error: 0.5396 Root Mean Square Error: 0.7181 Mean Square Error: 0.5157
model.save(MODELFILENAME+'.h5')
#càlcul del threshold de test
def calculate_threshold(X_test, X_test_pred):
distance = np.sqrt(np.mean(np.square(X_test_pred - X_test),axis=1))
"""Sorting the scores/diffs and using a 0.80 as cutoff value to pick the threshold"""
distance.sort();
cut_off = int(0.9 * len(distance));
threshold = distance[cut_off];
return threshold
for col in columns:
print ("####################### "+col +" ###########################")
#Standardize the test data
scaler = StandardScaler()
test_cpy = test.copy()
test[col] = scaler.fit_transform(test[[col]])
#creem seqüencia amb finestra temporal per les dades de test
X_test1, y_test1 = create_sequences(test[[col]], test[col])
print(f'Testing shape: {X_test1.shape}')
#evaluem el model
eval = model.evaluate(X_test1, y_test1)
print("evaluate: ",eval)
#predim el model
X_test1_pred = model.predict(X_test1, verbose=0)
evaluate_prediction(X_test1_pred, X_test1,"LSTM")
#càlcul del mae_loss
test1_mae_loss = np.mean(np.abs(X_test1_pred - X_test1), axis=1)
test1_rmse_loss = np.sqrt(np.mean(np.square(X_test1_pred - X_test1),axis=1))
# reshaping test prediction
X_test1_predReshape = X_test1_pred.reshape((X_test1_pred.shape[0] * X_test1_pred.shape[1]), X_test1_pred.shape[2])
# reshaping test data
X_test1Reshape = X_test1.reshape((X_test1.shape[0] * X_test1.shape[1]), X_test1.shape[2])
threshold_test = calculate_threshold(X_test1Reshape,X_test1_predReshape)
test1_score_df = pd.DataFrame(test[TIME_STEPS:])
test1_score_df['loss'] = test1_rmse_loss.reshape((-1))
test1_score_df['threshold'] = threshold_test
test1_score_df['anomaly'] = test1_score_df['loss'] > test1_score_df['threshold']
test1_score_df[col] = test[TIME_STEPS:][col]
#gràfic test lost i threshold
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['loss'], name='Test loss'))
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['threshold'], name='Threshold'))
fig.update_layout(showlegend=True, title='Test loss vs. Threshold')
fig.show()
#Posem les anomalies en un array
anomalies1 = test1_score_df.loc[test1_score_df['anomaly'] == True]
anomalies1.shape
print('anomalies: ',anomalies1.shape); print();
#Gràfic dels punts i de les anomalíes amb els valors de dades transformades per verificar que la normalització que s'ha fet no distorssiona les dades
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=scaler.inverse_transform(test1_score_df[col]), name=col))
fig.add_trace(go.Scatter(x=anomalies1.index, y=scaler.inverse_transform(anomalies1[col]), mode='markers', name='Anomaly'))
fig.update_layout(showlegend=True, title='Detected anomalies')
fig.show()
print ("######################################################")
####################### PM1 ########################### Testing shape: (774, 6, 1) 25/25 [==============================] - 0s 718us/step - loss: 0.6090 - rmse: 0.6303 - mse: 0.7243 evaluate: [0.6090001463890076, 0.6302738785743713, 0.7243474721908569] LSTM: Mean Absolute Error: 0.4962 Root Mean Square Error: 0.7446 Mean Square Error: 0.5544
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy test[col] = scaler.fit_transform(test[[col]])
anomalies: (74, 10)
###################################################### ####################### PM25 ########################### Testing shape: (774, 6, 1) 1/25 [>.............................] - ETA: 0s - loss: 0.7072 - rmse: 0.7525 - mse: 0.7091WARNING:tensorflow:Callbacks method `on_test_batch_begin` is slow compared to the batch time (batch time: 0.0000s vs `on_test_batch_begin` time: 0.0010s). Check your callbacks. 25/25 [==============================] - 0s 718us/step - loss: 0.6234 - rmse: 0.6447 - mse: 0.6954 evaluate: [0.6233577132225037, 0.6446775794029236, 0.695378839969635]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.5101 Root Mean Square Error: 0.7328 Mean Square Error: 0.5369
anomalies: (70, 10)
###################################################### ####################### PM10 ########################### Testing shape: (774, 6, 1) 25/25 [==============================] - 0s 638us/step - loss: 0.6440 - rmse: 0.6652 - mse: 0.6754 evaluate: [0.6440355181694031, 0.665185809135437, 0.6754423975944519] LSTM: Mean Absolute Error: 0.5316 Root Mean Square Error: 0.7156 Mean Square Error: 0.5121
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
anomalies: (72, 10)
###################################################### ####################### PM1ATM ########################### Testing shape: (774, 6, 1) 25/25 [==============================] - 0s 678us/step - loss: 0.6523 - rmse: 0.6739 - mse: 0.6940 evaluate: [0.6523045301437378, 0.6739341616630554, 0.694023847579956] LSTM: Mean Absolute Error: 0.5359 Root Mean Square Error: 0.7132 Mean Square Error: 0.5087
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
anomalies: (62, 10)
###################################################### ####################### PM25ATM ########################### Testing shape: (774, 6, 1) 25/25 [==============================] - 0s 758us/step - loss: 0.6456 - rmse: 0.6673 - mse: 0.6988 evaluate: [0.6455982327461243, 0.667287290096283, 0.69883793592453]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.5298 Root Mean Square Error: 0.7179 Mean Square Error: 0.5153
anomalies: (62, 10)
###################################################### ####################### PM10ATM ########################### Testing shape: (774, 6, 1) 25/25 [==============================] - 0s 718us/step - loss: 0.6324 - rmse: 0.6538 - mse: 0.6894 evaluate: [0.6324225068092346, 0.653846263885498, 0.6894000768661499]
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
LSTM: Mean Absolute Error: 0.5196 Root Mean Square Error: 0.7244 Mean Square Error: 0.5247
anomalies: (74, 10)
######################################################